this_notebook_name = "BreastSegmentationStudy-TF2"

# Update this folder name for your computer

local_data_folder = "/Usersß/Josh Ehrlich/Courses/CISC881/Project/data"
overwrite_existing_data_files = False

# All results and output will be archived with this timestamp

import datetime
save_timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
print("Save timestamp: {}".format(save_timestamp))

# Learning parameters

import numpy as np

ultrasound_size = 128
num_classes = 2
num_epochs = 200
batch_size = 64
max_learning_rate = 0.02
min_learning_rate = 0.000001
regularization_rate = 0.0001
filter_multiplier = 15
class_weights = np.array([0.15, 0.85])
learning_rate_decay = (max_learning_rate - min_learning_rate) / num_epochs

# Training data augmentation parameters

max_shift_factor = 0.12
max_rotation_angle = 10
max_zoom_factor = 1.1
min_zoom_factor = 0.8

# Evaluation parameters

acceptable_margin_mm = 1.0
mm_per_pixel = 1.0

roc_thresholds = [0.9, 0.8, 0.7, 0.65, 0.6, 0.55, 0.5, 0.45, 0.4, 0.35, 0.3, 0.25, 0.2, 0.15, 0.1,
                  0.08, 0.06, 0.04, 0.02, 0.01,
                  0.008, 0.006, 0.004, 0.002, 0.001]

'''
Provide NxM numpy array to schedule cross validation
N rounds of validation will be performed, leaving out M patients in each for validation data
All values should be valid patient IDs, or negative. Negative values are ignored.

Example 1: a leave-one-out cross validation with 3 patients would look like this:
validation_schedule_patient = np.array([[0],[1],[2]])

Example 2: a leave-two-out cross validation on 10 patients would look like this:
validation_schedule_patient = np.array([[0,1],[2,3],[4,5],[6,7],[8,9]])

Example 3: leave-one-out cross validation with 3 patients, then training on all available data (no validation):
validation_schedule_patient = np.array([[0],[1],[2],[-1]])
'''
validation_schedule_patient = np.array([[0,1,2,3], [7,11,13,18], [9,14,21,31], [19,22,29,32], [20,23,27,30]])

# Uncomment for faster debugging

roc_thresholds = [0.8, 0.6, 0.4, 0.2, 0.1, 0.01, 0.001]
num_epochs = 100

Save timestamp: 2020-12-10_17-15-10

import os
from random import sample
from pathlib import Path

from ipywidgets import IntProgress
from IPython.display import display, HTML

import diskcache
import girder_client
import matplotlib.pyplot as plt
import pandas as pd

import tensorflow as tf

import ultrasound_batch_generator as generator
import evaluation_metrics

from girder_apikey import girder_apikey

# Import aigt modules

import sys
parent_folder = os.path.dirname(os.path.abspath(os.curdir))
sys.path.append(parent_folder)

import Models.segmentation_unet as unet
import utils

# Creating standard folders to save data and logs

data_arrays_fullpath, notebooks_save_fullpath, results_save_fullpath, models_save_fullpath, val_data_fullpath =\
    utils.create_standard_project_folders(local_data_folder)

# Fetching Girder data

girder_url = "https://pocus.cs.queensu.ca/api/v1"
data_csv_file = "BreastGirder.csv"
girder_key = girder_apikey

ultrasound_arrays_by_patients, segmentation_arrays_by_patients =\
    utils.load_girder_data(data_csv_file, data_arrays_fullpath, girder_url, girder_key)
    
n_patients = len(ultrasound_arrays_by_patients)

for i in range(n_patients):
    print("Patient {} has {} ultrasounds and {} segmentations".format(
        i, ultrasound_arrays_by_patients[i].shape[0], segmentation_arrays_by_patients[i].shape[0]))

Patient 0 has 379 ultrasounds and 379 segmentations
Patient 1 has 292 ultrasounds and 292 segmentations
Patient 2 has 426 ultrasounds and 426 segmentations
Patient 3 has 418 ultrasounds and 418 segmentations
Patient 4 has 442 ultrasounds and 442 segmentations
Patient 5 has 224 ultrasounds and 224 segmentations
Patient 6 has 272 ultrasounds and 272 segmentations
Patient 7 has 288 ultrasounds and 288 segmentations
Patient 8 has 175 ultrasounds and 175 segmentations
Patient 9 has 113 ultrasounds and 113 segmentations
Patient 10 has 237 ultrasounds and 237 segmentations
Patient 11 has 180 ultrasounds and 180 segmentations
Patient 12 has 5 ultrasounds and 5 segmentations
Patient 13 has 402 ultrasounds and 402 segmentations
Patient 14 has 295 ultrasounds and 295 segmentations
Patient 15 has 331 ultrasounds and 331 segmentations
Patient 16 has 145 ultrasounds and 145 segmentations
Patient 17 has 253 ultrasounds and 253 segmentations
Patient 18 has 158 ultrasounds and 158 segmentations
Patient 19 has 187 ultrasounds and 187 segmentations
Patient 20 has 135 ultrasounds and 135 segmentations
Patient 21 has 107 ultrasounds and 107 segmentations
Patient 22 has 238 ultrasounds and 238 segmentations
Patient 23 has 226 ultrasounds and 226 segmentations
Patient 24 has 162 ultrasounds and 162 segmentations
Patient 25 has 124 ultrasounds and 124 segmentations
Patient 26 has 209 ultrasounds and 209 segmentations
Patient 27 has 123 ultrasounds and 123 segmentations
Patient 28 has 181 ultrasounds and 181 segmentations
Patient 29 has 134 ultrasounds and 134 segmentations
Patient 30 has 145 ultrasounds and 145 segmentations
Patient 31 has 133 ultrasounds and 133 segmentations
Patient 32 has 179 ultrasounds and 179 segmentations

# Prepare validation rounds

if np.max(np.max(validation_schedule_patient)) > (n_patients - 1):
    raise Exception("Patient ID cannot be greater than {}".format(n_patients - 1))

num_validation_rounds = len(validation_schedule_patient)
print("Planning {} rounds of validation".format(num_validation_rounds))
for i in range(num_validation_rounds):
    print("Validation on patients {} in round {}".format(validation_schedule_patient[i], i))

Planning 5 rounds of validation
Validation on patients [0 1 2 3] in round 0
Validation on patients [ 7 11 13 18] in round 1
Validation on patients [ 9 14 21 31] in round 2
Validation on patients [19 22 29 32] in round 3
Validation on patients [20 23 27 30] in round 4

# Print training parameters, to archive them together with the notebook output.

time_sequence_start = datetime.datetime.now()

print("Timestamp for saved files: {}".format(save_timestamp))
print("\nTraining parameters")
print("Number of epochs:    {}".format(num_epochs))
print("Step size maximum:   {}".format(max_learning_rate))
print("Step size decay:     {}".format(learning_rate_decay))
print("Batch size:          {}".format(batch_size))
print("Regularization rate: {}".format(regularization_rate))
print("")
print("Saving validation predictions in: {}".format(val_data_fullpath))
print("Saving models in:                 {}".format(models_save_fullpath))

# ROC data will be saved in these containers

val_best_metrics    = dict()
val_fuzzy_metrics   = dict()
val_aurocs          = np.zeros(num_validation_rounds)
val_best_thresholds = np.zeros(num_validation_rounds)

# Perform validation rounds

for val_round_index in range(num_validation_rounds):
    
    # Prepare data arrays
    
    train_ultrasound_data = np.zeros(
        [0,
         ultrasound_arrays_by_patients[0].shape[1],
         ultrasound_arrays_by_patients[0].shape[2],
         ultrasound_arrays_by_patients[0].shape[3]])
    
    train_segmentation_data = np.zeros(
        [0,
         segmentation_arrays_by_patients[0].shape[1],
         segmentation_arrays_by_patients[0].shape[2],
         segmentation_arrays_by_patients[0].shape[3]])
    
    val_ultrasound_data = np.zeros(
        [0,
         ultrasound_arrays_by_patients[0].shape[1],
         ultrasound_arrays_by_patients[0].shape[2],
         ultrasound_arrays_by_patients[0].shape[3]])
    
    val_segmentation_data = np.zeros(
        [0,
         segmentation_arrays_by_patients[0].shape[1],
         segmentation_arrays_by_patients[0].shape[2],
         segmentation_arrays_by_patients[0].shape[3]])
    
    for patient_index in range(n_patients):
        if patient_index not in validation_schedule_patient[val_round_index]:
            train_ultrasound_data = np.concatenate((train_ultrasound_data,
                                                    ultrasound_arrays_by_patients[patient_index]))
            train_segmentation_data = np.concatenate((train_segmentation_data,
                                                      segmentation_arrays_by_patients[patient_index]))
        else:
            val_ultrasound_data = np.concatenate((val_ultrasound_data,
                                                 ultrasound_arrays_by_patients[patient_index]))
            val_segmentation_data = np.concatenate((val_segmentation_data,
                                                   segmentation_arrays_by_patients[patient_index]))
    
    n_train = train_ultrasound_data.shape[0]
    n_val = val_ultrasound_data.shape[0]
    
    print("\n*** Leave-one-out round # {}".format(val_round_index))
    print("    Training on {} images, validating on {} images...".format(n_train, n_val))
    
    val_segmentation_data_onehot = tf.keras.utils.to_categorical(val_segmentation_data, num_classes)
    
    # Create and train model
    
    model = unet.segmentation_unet(ultrasound_size, num_classes, filter_multiplier, regularization_rate)
    
    model.compile(
        optimizer=tf.keras.optimizers.Adam(lr=max_learning_rate, decay=learning_rate_decay),
        loss=unet.weighted_categorical_crossentropy(class_weights),
        metrics=["accuracy"]
    )
    """
    #----------------------------------------------------------------------------
    import os
    import sys
    import datetime
    from random import sample
    from pathlib import Path
    import girder_client
    import matplotlib.pyplot as plt
    import pandas as pd
    import tensorflow as tf
    import numpy as np

    from Spine.ultrasound_batch_generator import train_preprocess, train_preprocess_with_maps, generate_weight_maps
    from Spine import evaluation_metrics

    from Spine.models import (
        new_unet,
        weighted_categorical_crossentropy,
        weighted_categorical_crossentropy_with_maps,
    )
    import utils


    batch_size=128, 
    num_epochs=100, 
    sagittal_only=False, 
    num_frames=1, 
    with_maps=False, 
    learning_rate=0.002,
    lr_decay=False,
    dropout=0.0,
    use_attention=True,
    num_layers=5,
    filters=16,
    use_batch_norm=True,
    load_from_save=False,

    ultrasound_size = 128
    batch_size = 128
    num_classes = 2
    min_learning_rate = 0.00001
    class_weights = np.array([0.1, 0.9])
    learning_rate_decay = (0.002-0.00001) / 100
    num_epochs = 100

    model = new_unet(
                input_size = ultrasound_size,
                num_classes=num_classes,
                num_channels=num_frames,
                use_batch_norm=use_batch_norm,
                upsample_mode="deconv",  # 'deconv' or 'simple'
                dropout=dropout,
                dropout_type="spatial",
                use_attention=use_attention,
                filters=filters,
                num_layers=num_layers,
                output_activation="softmax",
            )

    learning_rate = 0.002
    loss_func = weighted_categorical_crossentropy(class_weights)
    
    preprocess_func = train_preprocess
    print(learning_rate, learning_rate_decay)
    model.compile(
                optimizer=tf.keras.optimizers.Adam(
                    lr=learning_rate, decay=learning_rate_decay
                ),
                loss=loss_func,
                metrics=["accuracy", evaluation_metrics.jaccard_coef, evaluation_metrics.dice_coef],
            )

#-----------------------------------------------------
    """
    
    model.summary() #prints the structure of the neural network. If you change the unet,
    #print this so yuo can see it.

    training_generator = generator.UltrasoundSegmentationBatchGenerator(
        train_ultrasound_data,
        train_segmentation_data[:, :, :, 0],
        batch_size,
        (ultrasound_size, ultrasound_size),
        max_shift_factor=max_shift_factor,
        min_zoom_factor=min_zoom_factor,
        max_zoom_factor=max_zoom_factor,
        max_rotation_angle=max_rotation_angle
    )
        
    training_time_start = datetime.datetime.now()
    print("TRAINING LOG",training_generator)
    if n_val > 0:
        training_log = model.fit_generator(
            training_generator,
            validation_data=(val_ultrasound_data, val_segmentation_data_onehot),
            epochs=num_epochs,
            verbose=0)
    else:
        training_log = model.fit_generator(training_generator, epochs=num_epochs, verbose=0)
    
    training_time_stop = datetime.datetime.now()
    
    # Pring training log
    
    print("  Training time: {}".format(training_time_stop-training_time_start))
    
    # Plot training loss and metrics
    
    fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4))
    
    axes[0].plot(training_log.history['loss'], 'bo--')
    if n_val > 0:
        axes[0].plot(training_log.history['val_loss'], 'ro-')
    axes[0].set(xlabel='Epochs (n)', ylabel='Loss')
    if n_val > 0:
        axes[0].legend(['Training loss', 'Validation loss'])
    
    axes[1].plot(training_log.history['accuracy'], 'bo--')
    if n_val > 0:
        axes[1].plot(training_log.history['val_accuracy'], 'ro-')
    axes[1].set(xlabel='Epochs (n)', ylabel='Accuracy')
    if n_val > 0:
        axes[1].legend(['Training accuracy', 'Validation accuracy'])
    
    fig.tight_layout()
    
    # Archive trained model with unique filename based on notebook name and timestamp
    model_file_name = this_notebook_name + "_model-" + str(val_round_index) + "_" + save_timestamp + ".h5"
    model_fullname = os.path.join(models_save_fullpath, model_file_name)
    model.save(model_fullname)

    # Predict on validation data
    
    if n_val > 0:
        print(val_ultrasound_data.shape)
        y_pred_val  = model.predict(val_ultrasound_data)

        # Saving predictions for further evaluation

        val_prediction_filename = save_timestamp + "_prediction_" + str(val_round_index) + ".npy"
        val_prediction_fullname = os.path.join(val_data_fullpath, val_prediction_filename)
        np.save(val_prediction_fullname, y_pred_val)
        
        # Validation results

        vali_metrics_dicts, vali_best_threshold_index, vali_area = evaluation_metrics.compute_roc(
            roc_thresholds, y_pred_val, val_segmentation_data, acceptable_margin_mm, mm_per_pixel)

        val_fuzzy_metrics[val_round_index] = evaluation_metrics.compute_evaluation_metrics(
            y_pred_val, val_segmentation_data, acceptable_margin_mm, mm_per_pixel)

        val_best_metrics[val_round_index]    = vali_metrics_dicts[vali_best_threshold_index]
        val_aurocs[val_round_index]          = vali_area
        val_best_thresholds[val_round_index] = roc_thresholds[vali_best_threshold_index]
    
    # Printing total time of this validation round
    
    print("\nTotal round time:  {}".format(datetime.datetime.now() - training_time_start))
    print("")


time_sequence_stop = datetime.datetime.now()

print("\nTotal training time:   {}".format(time_sequence_stop - time_sequence_start))

Timestamp for saved files: 2020-12-10_17-15-10

Training parameters
Number of epochs:    100
Step size maximum:   0.02
Step size decay:     9.9995e-05
Batch size:          64
Regularization rate: 0.0001

Saving validation predictions in: /Usersß/Josh Ehrlich/Courses/CISC881/Project/data\PredictionsValidation
Saving models in:                 /Usersß/Josh Ehrlich/Courses/CISC881/Project/data\SavedModels

*** Leave-one-out round # 0
    Training on 5803 images, validating on 1515 images...
[<tf.Tensor 'conv2d_8/Relu:0' shape=(None, 4, 4, 128) dtype=float32>]
Model: "functional_1"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d (Conv2D)                 (None, 128, 128, 16) 160         input_1[0][0]                    
__________________________________________________________________________________________________
batch_normalization (BatchNorma (None, 128, 128, 16) 64          conv2d[0][0]                     
__________________________________________________________________________________________________
spatial_dropout2d (SpatialDropo (None, 128, 128, 16) 0           batch_normalization[0][0]        
__________________________________________________________________________________________________
conv2d_1 (Conv2D)               (None, 128, 128, 16) 2320        spatial_dropout2d[0][0]          
__________________________________________________________________________________________________
batch_normalization_1 (BatchNor (None, 128, 128, 16) 64          conv2d_1[0][0]                   
__________________________________________________________________________________________________
max_pooling2d (MaxPooling2D)    (None, 64, 64, 16)   0           batch_normalization_1[0][0]      
__________________________________________________________________________________________________
conv2d_2 (Conv2D)               (None, 64, 64, 32)   4640        max_pooling2d[0][0]              
__________________________________________________________________________________________________
batch_normalization_2 (BatchNor (None, 64, 64, 32)   128         conv2d_2[0][0]                   
__________________________________________________________________________________________________
spatial_dropout2d_1 (SpatialDro (None, 64, 64, 32)   0           batch_normalization_2[0][0]      
__________________________________________________________________________________________________
conv2d_3 (Conv2D)               (None, 64, 64, 32)   9248        spatial_dropout2d_1[0][0]        
__________________________________________________________________________________________________
batch_normalization_3 (BatchNor (None, 64, 64, 32)   128         conv2d_3[0][0]                   
__________________________________________________________________________________________________
max_pooling2d_1 (MaxPooling2D)  (None, 32, 32, 32)   0           batch_normalization_3[0][0]      
__________________________________________________________________________________________________
conv2d_4 (Conv2D)               (None, 32, 32, 64)   18496       max_pooling2d_1[0][0]            
__________________________________________________________________________________________________
batch_normalization_4 (BatchNor (None, 32, 32, 64)   256         conv2d_4[0][0]                   
__________________________________________________________________________________________________
spatial_dropout2d_2 (SpatialDro (None, 32, 32, 64)   0           batch_normalization_4[0][0]      
__________________________________________________________________________________________________
conv2d_5 (Conv2D)               (None, 32, 32, 64)   36928       spatial_dropout2d_2[0][0]        
__________________________________________________________________________________________________
batch_normalization_5 (BatchNor (None, 32, 32, 64)   256         conv2d_5[0][0]                   
__________________________________________________________________________________________________
max_pooling2d_2 (MaxPooling2D)  (None, 16, 16, 64)   0           batch_normalization_5[0][0]      
__________________________________________________________________________________________________
conv2d_6 (Conv2D)               (None, 16, 16, 128)  73856       max_pooling2d_2[0][0]            
__________________________________________________________________________________________________
batch_normalization_6 (BatchNor (None, 16, 16, 128)  512         conv2d_6[0][0]                   
__________________________________________________________________________________________________
spatial_dropout2d_3 (SpatialDro (None, 16, 16, 128)  0           batch_normalization_6[0][0]      
__________________________________________________________________________________________________
conv2d_7 (Conv2D)               (None, 16, 16, 128)  147584      spatial_dropout2d_3[0][0]        
__________________________________________________________________________________________________
batch_normalization_7 (BatchNor (None, 16, 16, 128)  512         conv2d_7[0][0]                   
__________________________________________________________________________________________________
max_pooling2d_3 (MaxPooling2D)  (None, 8, 8, 128)    0           batch_normalization_7[0][0]      
__________________________________________________________________________________________________
conv2d_8 (Conv2D)               (None, 4, 4, 128)    147584      max_pooling2d_3[0][0]            
__________________________________________________________________________________________________
conv2d_transpose (Conv2DTranspo (None, 8, 8, 58)     29754       conv2d_8[0][0]                   
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 8, 8, 186)    0           conv2d_transpose[0][0]           
                                                                 max_pooling2d_3[0][0]            
__________________________________________________________________________________________________
conv2d_9 (Conv2D)               (None, 8, 8, 58)     43210       concatenate[0][0]                
__________________________________________________________________________________________________
batch_normalization_8 (BatchNor (None, 8, 8, 58)     232         conv2d_9[0][0]                   
__________________________________________________________________________________________________
conv2d_10 (Conv2D)              (None, 8, 8, 58)     13514       batch_normalization_8[0][0]      
__________________________________________________________________________________________________
batch_normalization_9 (BatchNor (None, 8, 8, 58)     232         conv2d_10[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_1 (Conv2DTrans (None, 16, 16, 26)   6058        batch_normalization_9[0][0]      
__________________________________________________________________________________________________
concatenate_1 (Concatenate)     (None, 16, 16, 90)   0           conv2d_transpose_1[0][0]         
                                                                 max_pooling2d_2[0][0]            
__________________________________________________________________________________________________
conv2d_11 (Conv2D)              (None, 16, 16, 26)   9386        concatenate_1[0][0]              
__________________________________________________________________________________________________
batch_normalization_10 (BatchNo (None, 16, 16, 26)   104         conv2d_11[0][0]                  
__________________________________________________________________________________________________
conv2d_12 (Conv2D)              (None, 16, 16, 26)   2730        batch_normalization_10[0][0]     
__________________________________________________________________________________________________
batch_normalization_11 (BatchNo (None, 16, 16, 26)   104         conv2d_12[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_2 (Conv2DTrans (None, 32, 32, 10)   1050        batch_normalization_11[0][0]     
__________________________________________________________________________________________________
concatenate_2 (Concatenate)     (None, 32, 32, 42)   0           conv2d_transpose_2[0][0]         
                                                                 max_pooling2d_1[0][0]            
__________________________________________________________________________________________________
conv2d_13 (Conv2D)              (None, 32, 32, 10)   1690        concatenate_2[0][0]              
__________________________________________________________________________________________________
batch_normalization_12 (BatchNo (None, 32, 32, 10)   40          conv2d_13[0][0]                  
__________________________________________________________________________________________________
conv2d_14 (Conv2D)              (None, 32, 32, 10)   410         batch_normalization_12[0][0]     
__________________________________________________________________________________________________
batch_normalization_13 (BatchNo (None, 32, 32, 10)   40          conv2d_14[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_3 (Conv2DTrans (None, 64, 64, 2)    82          batch_normalization_13[0][0]     
__________________________________________________________________________________________________
concatenate_3 (Concatenate)     (None, 64, 64, 18)   0           conv2d_transpose_3[0][0]         
                                                                 max_pooling2d[0][0]              
__________________________________________________________________________________________________
conv2d_15 (Conv2D)              (None, 64, 64, 2)    146         concatenate_3[0][0]              
__________________________________________________________________________________________________
batch_normalization_14 (BatchNo (None, 64, 64, 2)    8           conv2d_15[0][0]                  
__________________________________________________________________________________________________
conv2d_16 (Conv2D)              (None, 64, 64, 2)    18          batch_normalization_14[0][0]     
__________________________________________________________________________________________________
batch_normalization_15 (BatchNo (None, 64, 64, 2)    8           conv2d_16[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_4 (Conv2DTrans (None, 128, 128, 2)  18          batch_normalization_15[0][0]     
__________________________________________________________________________________________________
conv2d_17 (Conv2D)              (None, 128, 128, 2)  18          conv2d_transpose_4[0][0]         
==================================================================================================
Total params: 551,588
Trainable params: 550,244
Non-trainable params: 1,344
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x0000024487812520>
WARNING:tensorflow:From <ipython-input-7-639783e879e1>:175: Model.fit_generator (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.
Instructions for updating:
Please use Model.fit, which supports generators.
  Training time: 4:50:14.250505
(1515, 128, 128, 1)

Total round time:  4:50:28.189714


*** Leave-one-out round # 1
    Training on 6290 images, validating on 1028 images...
[<tf.Tensor 'conv2d_26/Relu:0' shape=(None, 4, 4, 128) dtype=float32>]
Model: "functional_3"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_2 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_18 (Conv2D)              (None, 128, 128, 16) 160         input_2[0][0]                    
__________________________________________________________________________________________________
batch_normalization_16 (BatchNo (None, 128, 128, 16) 64          conv2d_18[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_4 (SpatialDro (None, 128, 128, 16) 0           batch_normalization_16[0][0]     
__________________________________________________________________________________________________
conv2d_19 (Conv2D)              (None, 128, 128, 16) 2320        spatial_dropout2d_4[0][0]        
__________________________________________________________________________________________________
batch_normalization_17 (BatchNo (None, 128, 128, 16) 64          conv2d_19[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_4 (MaxPooling2D)  (None, 64, 64, 16)   0           batch_normalization_17[0][0]     
__________________________________________________________________________________________________
conv2d_20 (Conv2D)              (None, 64, 64, 32)   4640        max_pooling2d_4[0][0]            
__________________________________________________________________________________________________
batch_normalization_18 (BatchNo (None, 64, 64, 32)   128         conv2d_20[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_5 (SpatialDro (None, 64, 64, 32)   0           batch_normalization_18[0][0]     
__________________________________________________________________________________________________
conv2d_21 (Conv2D)              (None, 64, 64, 32)   9248        spatial_dropout2d_5[0][0]        
__________________________________________________________________________________________________
batch_normalization_19 (BatchNo (None, 64, 64, 32)   128         conv2d_21[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_5 (MaxPooling2D)  (None, 32, 32, 32)   0           batch_normalization_19[0][0]     
__________________________________________________________________________________________________
conv2d_22 (Conv2D)              (None, 32, 32, 64)   18496       max_pooling2d_5[0][0]            
__________________________________________________________________________________________________
batch_normalization_20 (BatchNo (None, 32, 32, 64)   256         conv2d_22[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_6 (SpatialDro (None, 32, 32, 64)   0           batch_normalization_20[0][0]     
__________________________________________________________________________________________________
conv2d_23 (Conv2D)              (None, 32, 32, 64)   36928       spatial_dropout2d_6[0][0]        
__________________________________________________________________________________________________
batch_normalization_21 (BatchNo (None, 32, 32, 64)   256         conv2d_23[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_6 (MaxPooling2D)  (None, 16, 16, 64)   0           batch_normalization_21[0][0]     
__________________________________________________________________________________________________
conv2d_24 (Conv2D)              (None, 16, 16, 128)  73856       max_pooling2d_6[0][0]            
__________________________________________________________________________________________________
batch_normalization_22 (BatchNo (None, 16, 16, 128)  512         conv2d_24[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_7 (SpatialDro (None, 16, 16, 128)  0           batch_normalization_22[0][0]     
__________________________________________________________________________________________________
conv2d_25 (Conv2D)              (None, 16, 16, 128)  147584      spatial_dropout2d_7[0][0]        
__________________________________________________________________________________________________
batch_normalization_23 (BatchNo (None, 16, 16, 128)  512         conv2d_25[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_7 (MaxPooling2D)  (None, 8, 8, 128)    0           batch_normalization_23[0][0]     
__________________________________________________________________________________________________
conv2d_26 (Conv2D)              (None, 4, 4, 128)    147584      max_pooling2d_7[0][0]            
__________________________________________________________________________________________________
conv2d_transpose_5 (Conv2DTrans (None, 8, 8, 58)     29754       conv2d_26[0][0]                  
__________________________________________________________________________________________________
concatenate_4 (Concatenate)     (None, 8, 8, 186)    0           conv2d_transpose_5[0][0]         
                                                                 max_pooling2d_7[0][0]            
__________________________________________________________________________________________________
conv2d_27 (Conv2D)              (None, 8, 8, 58)     43210       concatenate_4[0][0]              
__________________________________________________________________________________________________
batch_normalization_24 (BatchNo (None, 8, 8, 58)     232         conv2d_27[0][0]                  
__________________________________________________________________________________________________
conv2d_28 (Conv2D)              (None, 8, 8, 58)     13514       batch_normalization_24[0][0]     
__________________________________________________________________________________________________
batch_normalization_25 (BatchNo (None, 8, 8, 58)     232         conv2d_28[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_6 (Conv2DTrans (None, 16, 16, 26)   6058        batch_normalization_25[0][0]     
__________________________________________________________________________________________________
concatenate_5 (Concatenate)     (None, 16, 16, 90)   0           conv2d_transpose_6[0][0]         
                                                                 max_pooling2d_6[0][0]            
__________________________________________________________________________________________________
conv2d_29 (Conv2D)              (None, 16, 16, 26)   9386        concatenate_5[0][0]              
__________________________________________________________________________________________________
batch_normalization_26 (BatchNo (None, 16, 16, 26)   104         conv2d_29[0][0]                  
__________________________________________________________________________________________________
conv2d_30 (Conv2D)              (None, 16, 16, 26)   2730        batch_normalization_26[0][0]     
__________________________________________________________________________________________________
batch_normalization_27 (BatchNo (None, 16, 16, 26)   104         conv2d_30[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_7 (Conv2DTrans (None, 32, 32, 10)   1050        batch_normalization_27[0][0]     
__________________________________________________________________________________________________
concatenate_6 (Concatenate)     (None, 32, 32, 42)   0           conv2d_transpose_7[0][0]         
                                                                 max_pooling2d_5[0][0]            
__________________________________________________________________________________________________
conv2d_31 (Conv2D)              (None, 32, 32, 10)   1690        concatenate_6[0][0]              
__________________________________________________________________________________________________
batch_normalization_28 (BatchNo (None, 32, 32, 10)   40          conv2d_31[0][0]                  
__________________________________________________________________________________________________
conv2d_32 (Conv2D)              (None, 32, 32, 10)   410         batch_normalization_28[0][0]     
__________________________________________________________________________________________________
batch_normalization_29 (BatchNo (None, 32, 32, 10)   40          conv2d_32[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_8 (Conv2DTrans (None, 64, 64, 2)    82          batch_normalization_29[0][0]     
__________________________________________________________________________________________________
concatenate_7 (Concatenate)     (None, 64, 64, 18)   0           conv2d_transpose_8[0][0]         
                                                                 max_pooling2d_4[0][0]            
__________________________________________________________________________________________________
conv2d_33 (Conv2D)              (None, 64, 64, 2)    146         concatenate_7[0][0]              
__________________________________________________________________________________________________
batch_normalization_30 (BatchNo (None, 64, 64, 2)    8           conv2d_33[0][0]                  
__________________________________________________________________________________________________
conv2d_34 (Conv2D)              (None, 64, 64, 2)    18          batch_normalization_30[0][0]     
__________________________________________________________________________________________________
batch_normalization_31 (BatchNo (None, 64, 64, 2)    8           conv2d_34[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_9 (Conv2DTrans (None, 128, 128, 2)  18          batch_normalization_31[0][0]     
__________________________________________________________________________________________________
conv2d_35 (Conv2D)              (None, 128, 128, 2)  18          conv2d_transpose_9[0][0]         
==================================================================================================
Total params: 551,588
Trainable params: 550,244
Non-trainable params: 1,344
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x00000244AF695A60>
  Training time: 5:10:34.646047
(1028, 128, 128, 1)

Total round time:  5:10:44.194631


*** Leave-one-out round # 2
    Training on 6670 images, validating on 648 images...
[<tf.Tensor 'conv2d_44/Relu:0' shape=(None, 4, 4, 128) dtype=float32>]
Model: "functional_5"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_3 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_36 (Conv2D)              (None, 128, 128, 16) 160         input_3[0][0]                    
__________________________________________________________________________________________________
batch_normalization_32 (BatchNo (None, 128, 128, 16) 64          conv2d_36[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_8 (SpatialDro (None, 128, 128, 16) 0           batch_normalization_32[0][0]     
__________________________________________________________________________________________________
conv2d_37 (Conv2D)              (None, 128, 128, 16) 2320        spatial_dropout2d_8[0][0]        
__________________________________________________________________________________________________
batch_normalization_33 (BatchNo (None, 128, 128, 16) 64          conv2d_37[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_8 (MaxPooling2D)  (None, 64, 64, 16)   0           batch_normalization_33[0][0]     
__________________________________________________________________________________________________
conv2d_38 (Conv2D)              (None, 64, 64, 32)   4640        max_pooling2d_8[0][0]            
__________________________________________________________________________________________________
batch_normalization_34 (BatchNo (None, 64, 64, 32)   128         conv2d_38[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_9 (SpatialDro (None, 64, 64, 32)   0           batch_normalization_34[0][0]     
__________________________________________________________________________________________________
conv2d_39 (Conv2D)              (None, 64, 64, 32)   9248        spatial_dropout2d_9[0][0]        
__________________________________________________________________________________________________
batch_normalization_35 (BatchNo (None, 64, 64, 32)   128         conv2d_39[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_9 (MaxPooling2D)  (None, 32, 32, 32)   0           batch_normalization_35[0][0]     
__________________________________________________________________________________________________
conv2d_40 (Conv2D)              (None, 32, 32, 64)   18496       max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
batch_normalization_36 (BatchNo (None, 32, 32, 64)   256         conv2d_40[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_10 (SpatialDr (None, 32, 32, 64)   0           batch_normalization_36[0][0]     
__________________________________________________________________________________________________
conv2d_41 (Conv2D)              (None, 32, 32, 64)   36928       spatial_dropout2d_10[0][0]       
__________________________________________________________________________________________________
batch_normalization_37 (BatchNo (None, 32, 32, 64)   256         conv2d_41[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_10 (MaxPooling2D) (None, 16, 16, 64)   0           batch_normalization_37[0][0]     
__________________________________________________________________________________________________
conv2d_42 (Conv2D)              (None, 16, 16, 128)  73856       max_pooling2d_10[0][0]           
__________________________________________________________________________________________________
batch_normalization_38 (BatchNo (None, 16, 16, 128)  512         conv2d_42[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_11 (SpatialDr (None, 16, 16, 128)  0           batch_normalization_38[0][0]     
__________________________________________________________________________________________________
conv2d_43 (Conv2D)              (None, 16, 16, 128)  147584      spatial_dropout2d_11[0][0]       
__________________________________________________________________________________________________
batch_normalization_39 (BatchNo (None, 16, 16, 128)  512         conv2d_43[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_11 (MaxPooling2D) (None, 8, 8, 128)    0           batch_normalization_39[0][0]     
__________________________________________________________________________________________________
conv2d_44 (Conv2D)              (None, 4, 4, 128)    147584      max_pooling2d_11[0][0]           
__________________________________________________________________________________________________
conv2d_transpose_10 (Conv2DTran (None, 8, 8, 58)     29754       conv2d_44[0][0]                  
__________________________________________________________________________________________________
concatenate_8 (Concatenate)     (None, 8, 8, 186)    0           conv2d_transpose_10[0][0]        
                                                                 max_pooling2d_11[0][0]           
__________________________________________________________________________________________________
conv2d_45 (Conv2D)              (None, 8, 8, 58)     43210       concatenate_8[0][0]              
__________________________________________________________________________________________________
batch_normalization_40 (BatchNo (None, 8, 8, 58)     232         conv2d_45[0][0]                  
__________________________________________________________________________________________________
conv2d_46 (Conv2D)              (None, 8, 8, 58)     13514       batch_normalization_40[0][0]     
__________________________________________________________________________________________________
batch_normalization_41 (BatchNo (None, 8, 8, 58)     232         conv2d_46[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_11 (Conv2DTran (None, 16, 16, 26)   6058        batch_normalization_41[0][0]     
__________________________________________________________________________________________________
concatenate_9 (Concatenate)     (None, 16, 16, 90)   0           conv2d_transpose_11[0][0]        
                                                                 max_pooling2d_10[0][0]           
__________________________________________________________________________________________________
conv2d_47 (Conv2D)              (None, 16, 16, 26)   9386        concatenate_9[0][0]              
__________________________________________________________________________________________________
batch_normalization_42 (BatchNo (None, 16, 16, 26)   104         conv2d_47[0][0]                  
__________________________________________________________________________________________________
conv2d_48 (Conv2D)              (None, 16, 16, 26)   2730        batch_normalization_42[0][0]     
__________________________________________________________________________________________________
batch_normalization_43 (BatchNo (None, 16, 16, 26)   104         conv2d_48[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_12 (Conv2DTran (None, 32, 32, 10)   1050        batch_normalization_43[0][0]     
__________________________________________________________________________________________________
concatenate_10 (Concatenate)    (None, 32, 32, 42)   0           conv2d_transpose_12[0][0]        
                                                                 max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
conv2d_49 (Conv2D)              (None, 32, 32, 10)   1690        concatenate_10[0][0]             
__________________________________________________________________________________________________
batch_normalization_44 (BatchNo (None, 32, 32, 10)   40          conv2d_49[0][0]                  
__________________________________________________________________________________________________
conv2d_50 (Conv2D)              (None, 32, 32, 10)   410         batch_normalization_44[0][0]     
__________________________________________________________________________________________________
batch_normalization_45 (BatchNo (None, 32, 32, 10)   40          conv2d_50[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_13 (Conv2DTran (None, 64, 64, 2)    82          batch_normalization_45[0][0]     
__________________________________________________________________________________________________
concatenate_11 (Concatenate)    (None, 64, 64, 18)   0           conv2d_transpose_13[0][0]        
                                                                 max_pooling2d_8[0][0]            
__________________________________________________________________________________________________
conv2d_51 (Conv2D)              (None, 64, 64, 2)    146         concatenate_11[0][0]             
__________________________________________________________________________________________________
batch_normalization_46 (BatchNo (None, 64, 64, 2)    8           conv2d_51[0][0]                  
__________________________________________________________________________________________________
conv2d_52 (Conv2D)              (None, 64, 64, 2)    18          batch_normalization_46[0][0]     
__________________________________________________________________________________________________
batch_normalization_47 (BatchNo (None, 64, 64, 2)    8           conv2d_52[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_14 (Conv2DTran (None, 128, 128, 2)  18          batch_normalization_47[0][0]     
__________________________________________________________________________________________________
conv2d_53 (Conv2D)              (None, 128, 128, 2)  18          conv2d_transpose_14[0][0]        
==================================================================================================
Total params: 551,588
Trainable params: 550,244
Non-trainable params: 1,344
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x00000244A79A9730>
  Training time: 5:26:57.929222
(648, 128, 128, 1)

Total round time:  5:27:04.321993


*** Leave-one-out round # 3
    Training on 6580 images, validating on 738 images...
[<tf.Tensor 'conv2d_62/Relu:0' shape=(None, 4, 4, 128) dtype=float32>]
Model: "functional_7"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_4 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_54 (Conv2D)              (None, 128, 128, 16) 160         input_4[0][0]                    
__________________________________________________________________________________________________
batch_normalization_48 (BatchNo (None, 128, 128, 16) 64          conv2d_54[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_12 (SpatialDr (None, 128, 128, 16) 0           batch_normalization_48[0][0]     
__________________________________________________________________________________________________
conv2d_55 (Conv2D)              (None, 128, 128, 16) 2320        spatial_dropout2d_12[0][0]       
__________________________________________________________________________________________________
batch_normalization_49 (BatchNo (None, 128, 128, 16) 64          conv2d_55[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_12 (MaxPooling2D) (None, 64, 64, 16)   0           batch_normalization_49[0][0]     
__________________________________________________________________________________________________
conv2d_56 (Conv2D)              (None, 64, 64, 32)   4640        max_pooling2d_12[0][0]           
__________________________________________________________________________________________________
batch_normalization_50 (BatchNo (None, 64, 64, 32)   128         conv2d_56[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_13 (SpatialDr (None, 64, 64, 32)   0           batch_normalization_50[0][0]     
__________________________________________________________________________________________________
conv2d_57 (Conv2D)              (None, 64, 64, 32)   9248        spatial_dropout2d_13[0][0]       
__________________________________________________________________________________________________
batch_normalization_51 (BatchNo (None, 64, 64, 32)   128         conv2d_57[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_13 (MaxPooling2D) (None, 32, 32, 32)   0           batch_normalization_51[0][0]     
__________________________________________________________________________________________________
conv2d_58 (Conv2D)              (None, 32, 32, 64)   18496       max_pooling2d_13[0][0]           
__________________________________________________________________________________________________
batch_normalization_52 (BatchNo (None, 32, 32, 64)   256         conv2d_58[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_14 (SpatialDr (None, 32, 32, 64)   0           batch_normalization_52[0][0]     
__________________________________________________________________________________________________
conv2d_59 (Conv2D)              (None, 32, 32, 64)   36928       spatial_dropout2d_14[0][0]       
__________________________________________________________________________________________________
batch_normalization_53 (BatchNo (None, 32, 32, 64)   256         conv2d_59[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_14 (MaxPooling2D) (None, 16, 16, 64)   0           batch_normalization_53[0][0]     
__________________________________________________________________________________________________
conv2d_60 (Conv2D)              (None, 16, 16, 128)  73856       max_pooling2d_14[0][0]           
__________________________________________________________________________________________________
batch_normalization_54 (BatchNo (None, 16, 16, 128)  512         conv2d_60[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_15 (SpatialDr (None, 16, 16, 128)  0           batch_normalization_54[0][0]     
__________________________________________________________________________________________________
conv2d_61 (Conv2D)              (None, 16, 16, 128)  147584      spatial_dropout2d_15[0][0]       
__________________________________________________________________________________________________
batch_normalization_55 (BatchNo (None, 16, 16, 128)  512         conv2d_61[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_15 (MaxPooling2D) (None, 8, 8, 128)    0           batch_normalization_55[0][0]     
__________________________________________________________________________________________________
conv2d_62 (Conv2D)              (None, 4, 4, 128)    147584      max_pooling2d_15[0][0]           
__________________________________________________________________________________________________
conv2d_transpose_15 (Conv2DTran (None, 8, 8, 58)     29754       conv2d_62[0][0]                  
__________________________________________________________________________________________________
concatenate_12 (Concatenate)    (None, 8, 8, 186)    0           conv2d_transpose_15[0][0]        
                                                                 max_pooling2d_15[0][0]           
__________________________________________________________________________________________________
conv2d_63 (Conv2D)              (None, 8, 8, 58)     43210       concatenate_12[0][0]             
__________________________________________________________________________________________________
batch_normalization_56 (BatchNo (None, 8, 8, 58)     232         conv2d_63[0][0]                  
__________________________________________________________________________________________________
conv2d_64 (Conv2D)              (None, 8, 8, 58)     13514       batch_normalization_56[0][0]     
__________________________________________________________________________________________________
batch_normalization_57 (BatchNo (None, 8, 8, 58)     232         conv2d_64[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_16 (Conv2DTran (None, 16, 16, 26)   6058        batch_normalization_57[0][0]     
__________________________________________________________________________________________________
concatenate_13 (Concatenate)    (None, 16, 16, 90)   0           conv2d_transpose_16[0][0]        
                                                                 max_pooling2d_14[0][0]           
__________________________________________________________________________________________________
conv2d_65 (Conv2D)              (None, 16, 16, 26)   9386        concatenate_13[0][0]             
__________________________________________________________________________________________________
batch_normalization_58 (BatchNo (None, 16, 16, 26)   104         conv2d_65[0][0]                  
__________________________________________________________________________________________________
conv2d_66 (Conv2D)              (None, 16, 16, 26)   2730        batch_normalization_58[0][0]     
__________________________________________________________________________________________________
batch_normalization_59 (BatchNo (None, 16, 16, 26)   104         conv2d_66[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_17 (Conv2DTran (None, 32, 32, 10)   1050        batch_normalization_59[0][0]     
__________________________________________________________________________________________________
concatenate_14 (Concatenate)    (None, 32, 32, 42)   0           conv2d_transpose_17[0][0]        
                                                                 max_pooling2d_13[0][0]           
__________________________________________________________________________________________________
conv2d_67 (Conv2D)              (None, 32, 32, 10)   1690        concatenate_14[0][0]             
__________________________________________________________________________________________________
batch_normalization_60 (BatchNo (None, 32, 32, 10)   40          conv2d_67[0][0]                  
__________________________________________________________________________________________________
conv2d_68 (Conv2D)              (None, 32, 32, 10)   410         batch_normalization_60[0][0]     
__________________________________________________________________________________________________
batch_normalization_61 (BatchNo (None, 32, 32, 10)   40          conv2d_68[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_18 (Conv2DTran (None, 64, 64, 2)    82          batch_normalization_61[0][0]     
__________________________________________________________________________________________________
concatenate_15 (Concatenate)    (None, 64, 64, 18)   0           conv2d_transpose_18[0][0]        
                                                                 max_pooling2d_12[0][0]           
__________________________________________________________________________________________________
conv2d_69 (Conv2D)              (None, 64, 64, 2)    146         concatenate_15[0][0]             
__________________________________________________________________________________________________
batch_normalization_62 (BatchNo (None, 64, 64, 2)    8           conv2d_69[0][0]                  
__________________________________________________________________________________________________
conv2d_70 (Conv2D)              (None, 64, 64, 2)    18          batch_normalization_62[0][0]     
__________________________________________________________________________________________________
batch_normalization_63 (BatchNo (None, 64, 64, 2)    8           conv2d_70[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_19 (Conv2DTran (None, 128, 128, 2)  18          batch_normalization_63[0][0]     
__________________________________________________________________________________________________
conv2d_71 (Conv2D)              (None, 128, 128, 2)  18          conv2d_transpose_19[0][0]        
==================================================================================================
Total params: 551,588
Trainable params: 550,244
Non-trainable params: 1,344
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x00000244A61B0370>
  Training time: 5:22:03.631152
(738, 128, 128, 1)

Total round time:  5:22:10.622688


*** Leave-one-out round # 4
    Training on 6689 images, validating on 629 images...
[<tf.Tensor 'conv2d_80/Relu:0' shape=(None, 4, 4, 128) dtype=float32>]
Model: "functional_9"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_5 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_72 (Conv2D)              (None, 128, 128, 16) 160         input_5[0][0]                    
__________________________________________________________________________________________________
batch_normalization_64 (BatchNo (None, 128, 128, 16) 64          conv2d_72[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_16 (SpatialDr (None, 128, 128, 16) 0           batch_normalization_64[0][0]     
__________________________________________________________________________________________________
conv2d_73 (Conv2D)              (None, 128, 128, 16) 2320        spatial_dropout2d_16[0][0]       
__________________________________________________________________________________________________
batch_normalization_65 (BatchNo (None, 128, 128, 16) 64          conv2d_73[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_16 (MaxPooling2D) (None, 64, 64, 16)   0           batch_normalization_65[0][0]     
__________________________________________________________________________________________________
conv2d_74 (Conv2D)              (None, 64, 64, 32)   4640        max_pooling2d_16[0][0]           
__________________________________________________________________________________________________
batch_normalization_66 (BatchNo (None, 64, 64, 32)   128         conv2d_74[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_17 (SpatialDr (None, 64, 64, 32)   0           batch_normalization_66[0][0]     
__________________________________________________________________________________________________
conv2d_75 (Conv2D)              (None, 64, 64, 32)   9248        spatial_dropout2d_17[0][0]       
__________________________________________________________________________________________________
batch_normalization_67 (BatchNo (None, 64, 64, 32)   128         conv2d_75[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_17 (MaxPooling2D) (None, 32, 32, 32)   0           batch_normalization_67[0][0]     
__________________________________________________________________________________________________
conv2d_76 (Conv2D)              (None, 32, 32, 64)   18496       max_pooling2d_17[0][0]           
__________________________________________________________________________________________________
batch_normalization_68 (BatchNo (None, 32, 32, 64)   256         conv2d_76[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_18 (SpatialDr (None, 32, 32, 64)   0           batch_normalization_68[0][0]     
__________________________________________________________________________________________________
conv2d_77 (Conv2D)              (None, 32, 32, 64)   36928       spatial_dropout2d_18[0][0]       
__________________________________________________________________________________________________
batch_normalization_69 (BatchNo (None, 32, 32, 64)   256         conv2d_77[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_18 (MaxPooling2D) (None, 16, 16, 64)   0           batch_normalization_69[0][0]     
__________________________________________________________________________________________________
conv2d_78 (Conv2D)              (None, 16, 16, 128)  73856       max_pooling2d_18[0][0]           
__________________________________________________________________________________________________
batch_normalization_70 (BatchNo (None, 16, 16, 128)  512         conv2d_78[0][0]                  
__________________________________________________________________________________________________
spatial_dropout2d_19 (SpatialDr (None, 16, 16, 128)  0           batch_normalization_70[0][0]     
__________________________________________________________________________________________________
conv2d_79 (Conv2D)              (None, 16, 16, 128)  147584      spatial_dropout2d_19[0][0]       
__________________________________________________________________________________________________
batch_normalization_71 (BatchNo (None, 16, 16, 128)  512         conv2d_79[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_19 (MaxPooling2D) (None, 8, 8, 128)    0           batch_normalization_71[0][0]     
__________________________________________________________________________________________________
conv2d_80 (Conv2D)              (None, 4, 4, 128)    147584      max_pooling2d_19[0][0]           
__________________________________________________________________________________________________
conv2d_transpose_20 (Conv2DTran (None, 8, 8, 58)     29754       conv2d_80[0][0]                  
__________________________________________________________________________________________________
concatenate_16 (Concatenate)    (None, 8, 8, 186)    0           conv2d_transpose_20[0][0]        
                                                                 max_pooling2d_19[0][0]           
__________________________________________________________________________________________________
conv2d_81 (Conv2D)              (None, 8, 8, 58)     43210       concatenate_16[0][0]             
__________________________________________________________________________________________________
batch_normalization_72 (BatchNo (None, 8, 8, 58)     232         conv2d_81[0][0]                  
__________________________________________________________________________________________________
conv2d_82 (Conv2D)              (None, 8, 8, 58)     13514       batch_normalization_72[0][0]     
__________________________________________________________________________________________________
batch_normalization_73 (BatchNo (None, 8, 8, 58)     232         conv2d_82[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_21 (Conv2DTran (None, 16, 16, 26)   6058        batch_normalization_73[0][0]     
__________________________________________________________________________________________________
concatenate_17 (Concatenate)    (None, 16, 16, 90)   0           conv2d_transpose_21[0][0]        
                                                                 max_pooling2d_18[0][0]           
__________________________________________________________________________________________________
conv2d_83 (Conv2D)              (None, 16, 16, 26)   9386        concatenate_17[0][0]             
__________________________________________________________________________________________________
batch_normalization_74 (BatchNo (None, 16, 16, 26)   104         conv2d_83[0][0]                  
__________________________________________________________________________________________________
conv2d_84 (Conv2D)              (None, 16, 16, 26)   2730        batch_normalization_74[0][0]     
__________________________________________________________________________________________________
batch_normalization_75 (BatchNo (None, 16, 16, 26)   104         conv2d_84[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_22 (Conv2DTran (None, 32, 32, 10)   1050        batch_normalization_75[0][0]     
__________________________________________________________________________________________________
concatenate_18 (Concatenate)    (None, 32, 32, 42)   0           conv2d_transpose_22[0][0]        
                                                                 max_pooling2d_17[0][0]           
__________________________________________________________________________________________________
conv2d_85 (Conv2D)              (None, 32, 32, 10)   1690        concatenate_18[0][0]             
__________________________________________________________________________________________________
batch_normalization_76 (BatchNo (None, 32, 32, 10)   40          conv2d_85[0][0]                  
__________________________________________________________________________________________________
conv2d_86 (Conv2D)              (None, 32, 32, 10)   410         batch_normalization_76[0][0]     
__________________________________________________________________________________________________
batch_normalization_77 (BatchNo (None, 32, 32, 10)   40          conv2d_86[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_23 (Conv2DTran (None, 64, 64, 2)    82          batch_normalization_77[0][0]     
__________________________________________________________________________________________________
concatenate_19 (Concatenate)    (None, 64, 64, 18)   0           conv2d_transpose_23[0][0]        
                                                                 max_pooling2d_16[0][0]           
__________________________________________________________________________________________________
conv2d_87 (Conv2D)              (None, 64, 64, 2)    146         concatenate_19[0][0]             
__________________________________________________________________________________________________
batch_normalization_78 (BatchNo (None, 64, 64, 2)    8           conv2d_87[0][0]                  
__________________________________________________________________________________________________
conv2d_88 (Conv2D)              (None, 64, 64, 2)    18          batch_normalization_78[0][0]     
__________________________________________________________________________________________________
batch_normalization_79 (BatchNo (None, 64, 64, 2)    8           conv2d_88[0][0]                  
__________________________________________________________________________________________________
conv2d_transpose_24 (Conv2DTran (None, 128, 128, 2)  18          batch_normalization_79[0][0]     
__________________________________________________________________________________________________
conv2d_89 (Conv2D)              (None, 128, 128, 2)  18          conv2d_transpose_24[0][0]        
==================================================================================================
Total params: 551,588
Trainable params: 550,244
Non-trainable params: 1,344
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x00000244A30890A0>
  Training time: 5:28:15.647152
(629, 128, 128, 1)

Total round time:  5:28:21.654835


Total training time:   1 day, 2:19:26.001064

# Arrange results in tables

metric_labels = [
    "AUROC",
    "best thresh",
    "best TP",
    "best FP",
    "best recall",
    "best precis",
    "fuzzy recall",
    "fuzzy precis",
    "fuzzy Fscore"
]

results_labels = []
  
for label in metric_labels:
    results_labels.append("Vali " + label)

results_df = pd.DataFrame(columns = results_labels)

for i in range(num_validation_rounds):
    if i in val_best_metrics.keys():
        results_df.loc[i] = [
            val_aurocs[i],
            val_best_thresholds[i],
            val_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE],
            val_best_metrics[i][evaluation_metrics.FALSE_POSITIVE_RATE],
            val_best_metrics[i][evaluation_metrics.RECALL],
            val_best_metrics[i][evaluation_metrics.PRECISION],
            val_fuzzy_metrics[i][evaluation_metrics.RECALL],
            val_fuzzy_metrics[i][evaluation_metrics.PRECISION],
            val_fuzzy_metrics[i][evaluation_metrics.FSCORE]
        ]

display(results_df)

print("\nAverages")

results_means_df = results_df.mean()
display(results_means_df)

	Vali AUROC	Vali best thresh	Vali best TP	Vali best FP	Vali best recall	Vali best precis	Vali fuzzy recall	Vali fuzzy precis	Vali fuzzy Fscore
0	0.993496	0.010	0.992773	0.040064	0.992773	0.172806	0.812573	0.454790	0.583179
1	0.759343	0.010	0.730836	0.130505	0.730836	0.066513	0.434268	0.220264	0.292281
2	0.967923	0.100	0.942393	0.058530	0.942393	0.262810	0.799042	0.361211	0.497517
3	0.784844	0.001	0.805391	0.118612	0.805391	0.169449	0.449002	0.523381	0.483347
4	0.866182	0.001	0.895561	0.137295	0.895561	0.187878	0.442975	0.569541	0.498348

Averages

Vali AUROC           0.874358
Vali best thresh     0.024400
Vali best TP         0.873391
Vali best FP         0.097001
Vali best recall     0.873391
Vali best precis     0.171891
Vali fuzzy recall    0.587572
Vali fuzzy precis    0.425837
Vali fuzzy Fscore    0.470934
dtype: float64

# Print the last ROC curve for visual verification that we catch the optimal point

n = len(roc_thresholds)

roc_x = np.zeros(n)
roc_y = np.zeros(n)

for i in range(n):
    roc_x[i] = vali_metrics_dicts[i][evaluation_metrics.FALSE_POSITIVE_RATE]
    roc_y[i] = vali_metrics_dicts[i][evaluation_metrics.SENSITIVITY]
    # print("Threshold = {0:4.2f}  False pos rate = {1:4.2f}  Sensitivity = {2:4.2f}"
    #       .format(roc_thresholds[i], roc_x[i], roc_y[i]))

    
plt.figure()
plt.ylim(-0.01, 1.01)
plt.xlim(-0.01, 1.01)
plt.plot(roc_x, roc_y, color='darkred', lw=2)

[<matplotlib.lines.Line2D at 0x244ad7bb1c0>]

# Save results table

csv_filename = this_notebook_name + "_" + save_timestamp + ".csv"
csv_fullname = os.path.join(results_save_fullpath, csv_filename)
results_df.to_csv(csv_fullname)

print("Results saved to: {}".format(csv_fullname))

Results saved to: /Usersß/Josh Ehrlich/Courses/CISC881/Project/data\SavedResults\BreastSegmentationStudy-TF2_2020-12-10_17-15-10.csv

# Display sample results

# determine the available ultrasounds in the last round of validation
last_round = validation_schedule_patient[-1]
print(val_ultrasound_data.shape)
prev_vali = 0

#20: [63, 94, 72, 121, 130, 2, 64, 111, 26, 46, 12, 67, 132, 49, 61, 90, 65, 106, 14, 100]
#23: [347, 185, 215, 228, 345, 176, 186, 335, 301, 237, 316, 288, 181, 354, 331, 261, 194, 225, 205, 153]
#27: [364, 459, 405, 464, 426, 397, 483, 416, 430, 392, 447, 401, 423, 409, 378, 458, 422, 445, 451, 379]
#30: [494, 488, 571, 626, 568, 529, 582, 486, 484, 533, 485, 574, 507, 516, 493, 499, 549, 561, 625, 523]
count = 0
patientNumber = [20, 23, 27, 30]
sample_indices = [[63, 94, 72, 121, 130, 2, 64, 111, 26, 46, 12, 67, 132, 49, 61, 90, 65, 106, 14, 100], [347, 185, 215, 228, 345, 176, 186, 335, 301, 237, 316, 288, 181, 354, 331, 261, 194, 225, 205, 153], [364, 459, 405, 464, 426, 397, 483, 416, 430, 392, 447, 401, 423, 409, 378, 458, 422, 445, 451, 379], [494, 488, 571, 626, 568, 529, 582, 486, 484, 533, 485, 574, 507, 516, 493, 499, 549, 561, 625, 523]]
for j in range(len(last_round)):
    
    num_vali =  prev_vali + ultrasound_arrays_by_patients[last_round[j]].shape[0]
    num_show = 20
    if num_vali < num_show:
        num_show = 0
    num_col = 4

    indices = [i for i in range(prev_vali, num_vali)]
    #sample_indices = sample(indices, num_show)
    print("Showing image slices for patient", patientNumber[count], ":", sample_indices[count])
    #sample_indices = [prev_vali]
    
    # update
    prev_vali = num_vali
    threshold = 0.5
    
    fig = plt.figure(figsize=(18, num_show*5))
    for i in range(num_show):
        a0 = fig.add_subplot(num_show, num_col, i*num_col+1)
        img0 = a0.imshow(np.flipud(val_ultrasound_data[sample_indices[count][i], :, :, 0].astype(np.float32)))
        a0.set_title("Patient #{} - Ultrasound #{}".format(last_round[j], sample_indices[count][i]))
        a1 = fig.add_subplot(num_show, num_col, i*num_col+2)
        img1 = a1.imshow(np.flipud(val_segmentation_data_onehot[sample_indices[count][i], :, :, 1]), vmin=0.0, vmax=1.0)
        a1.set_title("Segmentation #{}".format(sample_indices[count][i]))
        c = fig.colorbar(img1, fraction=0.046, pad=0.04)
        a2 = fig.add_subplot(num_show, num_col, i*num_col+3)
        img2 = a2.imshow(np.flipud(y_pred_val[sample_indices[count][i], :, :, 1]), vmin=0.0, vmax=1.0)
        a2.set_title("Prediction #{}".format(sample_indices[count][i]))
        c = fig.colorbar(img2, fraction=0.046, pad=0.04)
        a3 = fig.add_subplot(num_show, num_col, i*num_col+4)
        img3 = a3.imshow((np.flipud(y_pred_val[sample_indices[count][i], :, :, 1]) > threshold), vmin=0.0, vmax=1.0)
        c = fig.colorbar(img3, fraction=0.046, pad=0.04)
        a3.set_title("Thresholded #{}".format(sample_indices[count][i]))
    count = count + 1

(629, 128, 128, 1)
Showing image slices for patient 20 : [63, 94, 72, 121, 130, 2, 64, 111, 26, 46, 12, 67, 132, 49, 61, 90, 65, 106, 14, 100]
Showing image slices for patient 23 : [347, 185, 215, 228, 345, 176, 186, 335, 301, 237, 316, 288, 181, 354, 331, 261, 194, 225, 205, 153]
Showing image slices for patient 27 : [364, 459, 405, 464, 426, 397, 483, 416, 430, 392, 447, 401, 423, 409, 378, 458, 422, 445, 451, 379]
Showing image slices for patient 30 : [494, 488, 571, 626, 568, 529, 582, 486, 484, 533, 485, 574, 507, 516, 493, 499, 549, 561, 625, 523]

# Save notebook so all output is archived by the next cell

from IPython.display import Javascript
script = '''
require(["base/js/namespace"],function(Jupyter) {
    Jupyter.notebook.save_checkpoint();
});
'''
Javascript(script)

# Export HTML copy of this notebook

notebook_file_name =  "/Users/Josh Ehrlich/Courses/CISC881/Project/data/"+this_notebook_name + "_" + save_timestamp + ".html"
notebook_fullname = os.path.join(notebooks_save_fullpath, notebook_file_name)

os.system("jupyter nbconvert --to html " + this_notebook_name + " --output " + notebook_fullname)
print("Notebook saved to: {}".format(notebook_fullname))

Notebook saved to: /Users/Josh Ehrlich/Courses/CISC881/Project/data/BreastSegmentationStudy-TF2_2020-12-10_17-15-10.html

notebook_file_name

'/Users/Josh Ehrlich/Courses/CISC881/Project/data/BreastSegmentationStudy-TF2_2020-12-10_17-15-10.html'

print("Notebook_name", this_notebook_name)
print("fullname", notebook_fullname)

Notebook_name BreastSegmentationStudy-TF2
fullname /Users/Josh Ehrlich/Courses/CISC881/Project/data/BreastSegmentationStudy-TF2_2020-12-10_17-15-10.html

# Arrange results in tables

metric_labels = [
    'overall accuracy',
    'sensitivity-wrong',
    "AUROC",
    "best thresh",
    "best TP",
    "best FP",
    "best recall",
    "best precis",
    "fuzzy recall",
    "fuzzy precis",
    "fuzzy Fscore"
]

results_labels = []
  
for label in metric_labels:
    results_labels.append("Vali " + label)

results_df = pd.DataFrame(columns = results_labels)

for i in range(num_validation_rounds):
    if i in val_best_metrics.keys():
        results_df.loc[i] = [
            val_best_metrics[i][evaluation_metrics.ACCURACY]*100,
            (val_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE])/(val_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE] + (1-val_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE])),
            val_aurocs[i],
            val_best_thresholds[i],
            val_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE],
            val_best_metrics[i][evaluation_metrics.FALSE_POSITIVE_RATE],
            val_best_metrics[i][evaluation_metrics.RECALL],
            val_best_metrics[i][evaluation_metrics.PRECISION],
            val_fuzzy_metrics[i][evaluation_metrics.RECALL],
            val_fuzzy_metrics[i][evaluation_metrics.PRECISION],
            val_fuzzy_metrics[i][evaluation_metrics.FSCORE]
        ]

display(results_df)

print("\nAverages")

results_means_df = results_df.mean()
display(results_means_df)
'''

# Arrange results in tables

#metric_labels = [
#    "Acc",
#    "AUROC",
#    "best thresh",
#    "best TP",
#    "best FP",
#    "best recall",
#    "best precis",
#    "best dice",
#    "fuzzy recall",
#    "fuzzy precis",
#    "fuzzy Fscore"
#]

metric_labels = [
    "Overall Acc",
    "Image Classification Acc",
    "Img Classification TP",
    "Img Classification FP",
    "Img Classification FN",
    "Img Classification TN",
    "Img Classification Sensitivity (TP/(TP+FN))",
    "Img Classification Specificity (TN/(TN+FP))",
    "Img Classification Precision (TP/(TP+FP))",
    "Avg Center Distance",
    "Avg Prediction Overlap",
    "Avg Missed Prediction",
    "Dice"    
]
results_labels = []

for label in metric_labels:
    results_labels.append("Test " + label)

    
results_df = pd.DataFrame(columns = results_labels)

num_models = num_validation_rounds
test_best_metrics = val_best_metrics

for i in range(num_models):
    results_df.loc[i] = [
        test_best_metrics[i][evaluation_metrics.ACCURACY]*100,
        img_ACCs[i]*100,
        img_TPs[i],
        img_FPs[i],
        img_FNs[i],
        img_TNs[i],
        img_TPs[i]/(max(1,img_TPs[i]+img_FNs[i]))*100,
        img_TNs[i]/(max(1,img_TNs[i]+img_FPs[i]))*100,
        img_TPs[i]/(max(1,img_TPs[i]+img_FPs[i]))*100,
        totDistanceErrs[i],
        totSeg[i]*100,
        totMissed[i]*100,
        #test_aurocs[i],
        #test_best_thresholds[i],
        #test_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE],
        #test_best_metrics[i][evaluation_metrics.FALSE_POSITIVE_RATE],
        #test_best_metrics[i][evaluation_metrics.RECALL],
        #test_best_metrics[i][evaluation_metrics.PRECISION],
        test_best_metrics[i][evaluation_metrics.DICE]*100,
        #test_fuzzy_metrics[i][evaluation_metrics.RECALL],
        #test_fuzzy_metrics[i][evaluation_metrics.PRECISION],
        #test_fuzzy_metrics[i][evaluation_metrics.FSCORE],
    ]

display(results_df)

print("\nAverages")

results_means_df = results_df.mean()
results_std_df = results_df.std()
display(results_means_df)

print("\STD")
display(results_std_df)'''

	Vali overall accuracy	Vali sensitivity-wrong	Vali AUROC	Vali best thresh	Vali best TP	Vali best FP	Vali best recall	Vali best precis	Vali fuzzy recall	Vali fuzzy precis	Vali fuzzy Fscore
0	96.021036	0.992773	0.993496	0.010	0.992773	0.040064	0.992773	0.172806	0.812573	0.454790	0.583179
1	86.775252	0.730836	0.759343	0.010	0.730836	0.130505	0.730836	0.066513	0.434268	0.220264	0.292281
2	94.149037	0.942393	0.967923	0.100	0.942393	0.058530	0.942393	0.262810	0.799042	0.361211	0.497517
3	87.917067	0.805391	0.784844	0.001	0.805391	0.118612	0.805391	0.169449	0.449002	0.523381	0.483347
4	86.383018	0.895561	0.866182	0.001	0.895561	0.137295	0.895561	0.187878	0.442975	0.569541	0.498348

Averages

Vali overall accuracy     90.249082
Vali sensitivity-wrong     0.873391
Vali AUROC                 0.874358
Vali best thresh           0.024400
Vali best TP               0.873391
Vali best FP               0.097001
Vali best recall           0.873391
Vali best precis           0.171891
Vali fuzzy recall          0.587572
Vali fuzzy precis          0.425837
Vali fuzzy Fscore          0.470934
dtype: float64

'\n\n# Arrange results in tables\n\n#metric_labels = [\n#    "Acc",\n#    "AUROC",\n#    "best thresh",\n#    "best TP",\n#    "best FP",\n#    "best recall",\n#    "best precis",\n#    "best dice",\n#    "fuzzy recall",\n#    "fuzzy precis",\n#    "fuzzy Fscore"\n#]\n\nmetric_labels = [\n    "Overall Acc",\n    "Image Classification Acc",\n    "Img Classification TP",\n    "Img Classification FP",\n    "Img Classification FN",\n    "Img Classification TN",\n    "Img Classification Sensitivity (TP/(TP+FN))",\n    "Img Classification Specificity (TN/(TN+FP))",\n    "Img Classification Precision (TP/(TP+FP))",\n    "Avg Center Distance",\n    "Avg Prediction Overlap",\n    "Avg Missed Prediction",\n    "Dice"    \n]\nresults_labels = []\n\nfor label in metric_labels:\n    results_labels.append("Test " + label)\n\n    \nresults_df = pd.DataFrame(columns = results_labels)\n\nnum_models = num_validation_rounds\ntest_best_metrics = val_best_metrics\n\nfor i in range(num_models):\n    results_df.loc[i] = [\n        test_best_metrics[i][evaluation_metrics.ACCURACY]*100,\n        img_ACCs[i]*100,\n        img_TPs[i],\n        img_FPs[i],\n        img_FNs[i],\n        img_TNs[i],\n        img_TPs[i]/(max(1,img_TPs[i]+img_FNs[i]))*100,\n        img_TNs[i]/(max(1,img_TNs[i]+img_FPs[i]))*100,\n        img_TPs[i]/(max(1,img_TPs[i]+img_FPs[i]))*100,\n        totDistanceErrs[i],\n        totSeg[i]*100,\n        totMissed[i]*100,\n        #test_aurocs[i],\n        #test_best_thresholds[i],\n        #test_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE],\n        #test_best_metrics[i][evaluation_metrics.FALSE_POSITIVE_RATE],\n        #test_best_metrics[i][evaluation_metrics.RECALL],\n        #test_best_metrics[i][evaluation_metrics.PRECISION],\n        test_best_metrics[i][evaluation_metrics.DICE]*100,\n        #test_fuzzy_metrics[i][evaluation_metrics.RECALL],\n        #test_fuzzy_metrics[i][evaluation_metrics.PRECISION],\n        #test_fuzzy_metrics[i][evaluation_metrics.FSCORE],\n    ]\n\ndisplay(results_df)\n\nprint("\nAverages")\n\nresults_means_df = results_df.mean()\nresults_std_df = results_df.std()\ndisplay(results_means_df)\n\nprint("\\STD")\ndisplay(results_std_df)'